package com.kongchengji.spider.station.start.startImpl;

import com.kongchengji.spider.station.constant.Constant;
import com.kongchengji.spider.station.pipeline.StationPipeline;
import com.kongchengji.spider.station.process.processImpl.StationProcess;
import com.kongchengji.spider.station.start.StationInit;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.scheduler.BloomFilterDuplicateRemover;
import us.codecraft.webmagic.scheduler.QueueScheduler;

public class CommonStationInit implements StationInit{
    private Constant.Station station;
    private boolean isDirectlyGetTitle;
    private boolean isFindPage;
    private int sleepTime = 0;
    private int threadCount = 1;
    public CommonStationInit(int threadCount, Constant.Station station, boolean isDirectlyGetTitle, boolean isFindPage, int sleepTime){
        this.station = station;
        this.sleepTime = sleepTime;
        this.isDirectlyGetTitle = isDirectlyGetTitle;
        this.isFindPage = isFindPage;
        this.threadCount = threadCount;
    }
    public CommonStationInit(Constant.Station station, boolean isDirectlyGetTitle, boolean isFindPage){
        this.station = station;
        this.isDirectlyGetTitle = isDirectlyGetTitle;
        this.isFindPage = isFindPage;
    }
    public void run() {
        StationProcess processor = new StationProcess(station,isDirectlyGetTitle,isFindPage);
        processor.setSleepTime(sleepTime);
        Spider spider = Spider.create(processor).thread(threadCount);
        spider.addUrl(station.getUrl());
        spider.addPipeline(new StationPipeline(station));
        spider.setScheduler(new QueueScheduler().setDuplicateRemover(new BloomFilterDuplicateRemover(1000000)));
        spider.run();
    }


}
